package org.nlp.algo.classifier;

import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import org.apache.log4j.Logger;
import org.nlp.algo.util.HideFilter;

import com.aliasi.spell.TfIdfDistance;
import com.aliasi.tokenizer.NGramTokenizerFactory;
import com.aliasi.tokenizer.TokenizerFactory;
import com.aliasi.util.Files;

/**
 * @author longkeyy
 * 
 */
public class TrainTfIdf {
	private static Logger _log = Logger.getLogger(TrainTfIdf.class);
	// 语料文件夹
	private static String corpusPath = "resource/corpus";
		
	// 训练语料文件
	private static File trainDir = new File(corpusPath + "/train");
			
	// 定义分类
	private static String[] CATEGORIES;// = { "汽车", "财经", "IT", "健康", "体育",
										// "旅游", "教育", "招聘", "文化", "军事"};

	public static void main(String[] args) throws IOException {
		// TokenizerFactory tokenizerFactory =
		// IndoEuropeanTokenizerFactory.INSTANCE;
		// TokenizerFactory tokenizerFactory = LuceneTokenizerFactory.INSTANCE;
		TokenizerFactory tokenizerFactory = new NGramTokenizerFactory(2, 5);
		
		
		int catgoriesCount = 0;
		List<String> catgroies = new ArrayList<String>();
		File[] tranTypes = trainDir.listFiles(new FileFilter() {
			@Override
			public boolean accept(File arg0) {
				if (arg0.getName().startsWith(".")) {
					return false;
				}
				return true;
			}

		});
		for (int i = 0; i < tranTypes.length; i++) {
			if (tranTypes[i].isDirectory()
					&& !tranTypes[i].getName().startsWith(".")) {
				catgroies.add(tranTypes[i].toString().replace(
						tranTypes[i].getParent() + "\\", ""));
				_log.debug(tranTypes[catgoriesCount]);
				catgoriesCount++;
			}
		}

		CATEGORIES = new String[catgroies.size()];
		catgroies.toArray(CATEGORIES);

		TfIdfDistance classifier = new TfIdfDistance(tokenizerFactory);

		for (int i = 0; i < catgroies.size(); i++) {
			File classDir = new File(trainDir, CATEGORIES[i]);
			if (!classDir.isDirectory()) {
				_log.error("不能找到目录=" + classDir);
			}

			// 训练器遍历分类文件夹下的所有文件
			for (File files : classDir.listFiles(new HideFilter())) {
				String text = Files.readFromFile(files, "gbk");
				_log.debug("正在训练 " + CATEGORIES[i] + files.getName());
				classifier.handle(text);
			}
		}
				
		// 显示词频
		System.out.printf("\n\t%18s\t%8s\t%8s\n", "Term", "Doc Freq", "IDF");
		for (String term : classifier.termSet())
			System.out.printf("\t%18s\t%8d\t%8.2f\n", term,
					classifier.docFrequency(term), classifier.idf(term));

	}


}
